﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Text;
using System.Net;
using System.IO;
using HtmlAgilityPack;
using System.Text.RegularExpressions;
using System.Threading;

namespace ConsolidationTool.Common
{

    public class CrawlEngine
    { 
        public string URL { get; set; }

        public string Title { get; set; }

        public string OriginalUrl { get; set; }
 
        public string Crawl(string source)
        {

            var document = LoadPage(source);
            if (document == null) return string.Empty;

            var node = document.GetElementbyId("fs_commontable");

            return node.OuterHtml; 
        }

        private string RemoveRub(string p)
        {
            return p.Replace("/", "").Replace("http://","");
        }
        

        private HtmlDocument LoadPage(string pageSource)
        {
 
            HtmlDocument dochtml = new HtmlDocument();
     
            if (pageSource.IsNullOrEmpty()) return null;

            dochtml.LoadHtml(pageSource);
            while (dochtml.DocumentNode.ChildNodes.Count == 0)
            {
                dochtml.LoadHtml(GetPage(URL));
                Thread.Sleep(500);
            }
            return dochtml;             
        }

        public string GetPage(string url)
        {

            HttpWebRequest request = null;
            HttpWebResponse response = null;
            StreamReader reader = null;
            try
            {
                request = (HttpWebRequest)WebRequest.Create(url); 
            //    request.Timeout = 20000;
                request.AllowAutoRedirect = false;

                CredentialCache credentials = new CredentialCache();

                //credentials.Add(new Uri("https://decisions.court.gov.il/"), "NTLM", new NetworkCredential(@"idc\user8", "Password1"));
                request.UseDefaultCredentials = true;
                request.PreAuthenticate = true;
                request.Credentials = CredentialCache.DefaultCredentials;

                //credentials.Add("reqdev", 80, "NTLM", new NetworkCredential(@"reqeutte\mike", "`1qaz2wsx"));
                //request.Credentials = credentials;
                //request.Method = "GET";
                //request.KeepAlive = true;

                response = (HttpWebResponse)request.GetResponse(); 
                while(response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 512)
                { 
                    reader = new StreamReader(response.GetResponseStream(), Encoding.UTF8);
                    string html = reader.ReadToEnd();

                    return html;
                }
            }
            catch
            {
            }

            finally
            {
                if (response != null)
                { 
                    response.Close(); 
                    response = null; 
                }
                if (reader != null) 
                    reader.Close();

                if (request != null) 
                    request = null;

            }

            return string.Empty;

        }



    }
}